# coding=utf-8
import sys
import os
import pymongo
import json
import pytoml
import getopt
import traceback
import time
from thrift.protocol.TBinaryProtocol import TBinaryProtocol
from thrift.transport.TTransport import TMemoryBuffer
from select_from_mongo import SelectMongo
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from  background.models_sqlalchemy import Settings, Topic
from tld import get_tld
sys.path.append('..')
from i_util.pybeanstalk import PyBeanstalk
from bdp.i_crawler.i_downloader.ttypes import DownLoadReq
from i_util.pymysql import PyMySQL
from i_util.logs import LogHandler
from i_util.tools import str_obj, unicode_obj, get_md5_i64, extract_hzpost_url
import pickle

class Selector():
    def __init__(self, log, conf):
        self.logger = log
        self.conf = conf
        self.mysql_conf = conf['mysql_conf']['setting_config_database']
        #初始化linkbase
        self.mongodb_linkbase = self.mongo_config("linkbase");
        self.beanstalk_conf = conf.get('beanstalk_conf', {})
        self.beanstalk_client = self.create_beanstalk(self.beanstalk_conf)
    
    def mongo_config(self, mongo_item = "mongodb"):
        try:
            mongodb_conf = {}
            engine = create_engine(self.mysql_conf)
            Settings.metadata.create_all(engine)
            self.Dsession = sessionmaker(bind=engine)
            session = self.Dsession()
            query = session.query(Settings)
            records = query.filter(Settings.item == mongo_item).all()
            for record in records:
                mongodb_conf['host'] = record.value['host']
                mongodb_conf['port'] = int(record.value['port'])
                mongodb_conf['database'] = record.value['database']
                mongodb_conf['username'] = record.value['user']
                mongodb_conf['password'] = record.value['password']
                break;
            session.close();
            """
            mongo_conn = pymongo.MongoClient(
                host=mongodb_conf['host'],
                port=mongodb_conf['port']
            )
            mongo_db = mongo_conn[mongodb_conf['database']]
            username = mongodb_conf.get('username','')
            password = mongodb_conf.get('password','')
            if username and password:
                mongo_db.authenticate(username, password)
            """
            return mongodb_conf;
        except Exception as e:
            self.logger.error(traceback.format_exc())
            os._exit(1)

    def create_beanstalk(self, beanstalk_conf):
        try:
            beanstalk_client = PyBeanstalk(beanstalk_conf['host'], beanstalk_conf['port'])
            return beanstalk_client
        except:
            self.logger.error(traceback.format_exc())
            return None

    def create_download_req(self, url, link_attr, method = 'simple'):
        download_req = DownLoadReq()
        download_req.url = url
        download_req.post_data = {}
        download_req.src_type = 'linkbase'
        download_req.download_type = method
        scheduler_info = {}
        scheduler_info["schedule_time"] = time.time()
        download_req.scheduler = json.dumps(scheduler_info)
        return download_req

    def req_to_string(self, req):
        str_req = ""
        try:
            tMemory_b = TMemoryBuffer()
            tBinaryProtocol_b = TBinaryProtocol(tMemory_b)
            req.write(tBinaryProtocol_b)
            str_req = tMemory_b.getvalue() 
        except:
            self.logger.error('crawled_failt\terror:%s' % (traceback.format_exc()))
            #print traceback.format_exc()
        return str_req

    def select_links(self, site, url_format = '', index = False, method = 'simple'):
        beanstalk_client = self.create_beanstalk(self.beanstalk_conf)
        processor = SelectMongo(self.logger, self.mongodb_linkbase)
        home_page = "http://" + site + "/"
        domain = site
        try:
            domain = get_tld(home_page)
        except:
            self.logger.error('failt_domain\terror:%s' % (traceback.format_exc()))
        print domain, url_format
        items = processor.select_by_url_format(domain, url_format)
        num = 0
        for item in items:
            link_attr = pickle.loads(item['link_attr'])
            link      = item['url']
            if item.has_key('long_url'):
                link = item['long_url']
            status    = int(item.get('status', 0))
            need_crawl = 1;
            if link_attr.__dict__.has_key('extract_message') and link_attr.extract_message and link_attr.extract_message.ex_status == 2:
                need_crawl = 0
            dead_num = 0
            if need_crawl == 1 and link_attr and link_attr.normal_crawl_his:
                for crawl_his in link_attr.normal_crawl_his:
                    if crawl_his.status_code == 0:
                        need_crawl = 0
                        break
                    if crawl_his.http_code == 404 or crawl_his.http_code == 403:
                        dead_num += 1
            if dead_num >= 3 :
                need_crawl = 0
            if index:
                need_crawl = 1
            if need_crawl == 1:
                num += 1
                down_req =  self.create_download_req(link, link_attr, method)
                str_req = self.req_to_string(down_req)
                beanstalk_client.put(self.beanstalk_conf['output_tube'], str_req)
                print link, num
    def to_mq(self, link, link_attr, method = 'simple'):
        down_req =  self.create_download_req(link, link_attr, method)
        down_req.http_header = {};
        #down_req.http_header['Cookie'] = 
        #'FSSBBIl1UgzbN7N80S=yVfYl1nGrNBRaqXLSx7Gd2EL3kq9kTT9NLWPcTJLXgpt_5_FUTFVb_r8RaQc5mSi; ASP.NET_SessionId=xe2ustp1ylkxzj0jzm13dlwr; wafenterurl=L0luZGV4L0dldEFsbENvdW50P01tRXdNRD0xYWpCOXp2Q1lvV1FjSUxaemNfQmk0Ui5PZ19DSWRFb0owcVAzNFEubXdoYmc4WTFNWUNBV2U4MTdmMl9yazI2VHIuTnMuUzdNQnBoR2t4NjVFNzZmVFlhcUZYdGozZm14TWtvc20wd2E3QlVtbm81RmFfVWZUZHpIbEgwVkZRLkRXN0FFb1d1cFRSeF90dkZBWWNwM0lCRkV1bFdRRlE1YzdxQWhQczdkWVgwMTY2VFA1QkhhMFFNYkwxSjlXRTB0czB4Qk1zeVlpaFVBQkYuS05RNWE3bi4yazkwXzZSX3A4TE1ROG5aMjNLQUVWek5WOFQybWVpSHJUUGN6U0pXQVYwQmI5U0RZTXczQ2RlX1EuVFpzYkJENDZNWGJjaGNGY0pmOG5kMXAyQ0Z5MFN5bVpYaGdpNlpDbkdueWdTR3ZGQW4zMkpYMHFiLk5tN1JFa0I2eWhzVnF0NlNlZ3lOd2JTMEh4aTFmdm8yaHNwSVlYbWFnX3ZEbFRmdl9MaG1jWjM=; wafcookie=75a537d0c1f585098a22b5562d8af765; wafverify=c5f3c81f9c8bddf58a3e410b14d3c347; __utma=61363882.374222626.1484637197.1484637197.1484660404.2; __utmb=61363882.5.10.1484660404; __utmc=61363882; __utmz=61363882.1484637197.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); _gsref_2116842793=http://wenshu.court.gov.cn/; Hm_lvt_3f1a54c5a86d62407544d433f6418ef5=1484631796; Hm_lpvt_3f1a54c5a86d62407544d433f6418ef5=1484661000; _gscu_2116842793=84631795wg7im576; _gscs_2116842793=t84659778r5740g19|pv:10; _gscbrs_2116842793=1; FSSBBIl1UgzbN7N80T=1rCgfQ4fACAhoX0.LubghtmZbxbfFiVAEmQTwtqZq7USeBxm89joxWcmNAyzlnyGaa65KFz488JQ2nYG4yaG9lx9PCZBHE0.EoDe47b9wfyUUj5RWH9x45SdKukLymMcPh.4qyI41Pf1x6wrYJHl.K7QO9B.KXI2ulnNjg3rqLdKcy4g9JjjXiKe00coXUMO.bLG0hFsFKuN.rXHbR7TZ5xRLPb2ieeM.13ymxUjGiLlHj1PZO26qVzcx_neRdvhQJlSvQE083TKGNXw0wtBwg3OytIqVBk5GnWsNc48idTm6rnvzE_9sOIRIw1PrkkpafG' 
        str_req = self.req_to_string(down_req)
        self.beanstalk_client.put(self.beanstalk_conf['output_tube'], str_req)

def rule_sheduler():
    try:
        file_path = './scheduler.toml'
        opt, args = getopt.getopt(sys.argv[2:], 'f:', ['help'])
        for name, value in opt:
            if name == "-f":
                file_path = value
            elif name in ("-h", "--help"):
                usage()
                sys.exit()
            else:
                assert False, "unhandled option"

        with open(file_path, 'rb') as config:
            conf = pytoml.load(config)
            conf['log']=LogHandler(conf['server']['name']+str(conf['server']['port']))
            selecotr = Selector(conf['log'], conf)
            file_path =  "wenshu_url_failt.txt"
            if conf.has_key('parse_file') and conf['parse_file'].has_key('file_path'):
                file_path = conf['parse_file']['file_path']
            parse_file = open(file_path)
            for line in parse_file:
                method = 'simple'
                pars = line.strip().split('\t')
                if len(pars) < 3:
                    continue
                is_ready = int(pars[0])
                site = pars[1]
                url_format = pars[2]
                index = False
                if len(pars) >= 4 and pars[3] == "1":
                    index = True
                if len(pars) >= 5:
                    method =  pars[4]
                if is_ready == 1:
                    print url_format
                    selecotr.select_links(site, url_format, index, method)
    except getopt.GetoptError:
        print traceback.format_exc()
        sys.exit()

def file_sheduler():
    try:
        file_path = './scheduler.toml'
        opt, args = getopt.getopt(sys.argv[2:], 'f:', ['help'])
        for name, value in opt:
            if name == "-f":
                file_path = value
            elif name in ("-h", "--help"):
                usage()
                sys.exit()
            else:
                assert False, "unhandled option"

        with open(file_path, 'rb') as config:
            conf = pytoml.load(config)
            conf['log']=LogHandler(conf['server']['name']+str(conf['server']['port']))
            selecotr = Selector(conf['log'], conf)
            file_path =  "wenshu_url_failt.txt"
            if conf.has_key('parse_file') and conf['parse_file'].has_key('file_path'):
                file_path = conf['parse_file']['file_url']
            num = 0
            parse_file = open(file_path)
            for line in parse_file:
                pars = line.strip().split('\t')
                method = 'simple'
                hzurl = pars[0]
                if len(pars) == 2:
                    method = pars[1]
                postdata = extract_hzpost_url(hzurl).get('postdata', {})
                if postdata and (postdata.get('Param', '').find("2016-1") > 0 or postdata.get('Param', '').find("2017-01") > 0):
                    index = int(postdata.get('Index', '0'))
                    if index == 0:
                        continue
                    num += 1
                    selecotr.to_mq(hzurl, None, method)
                    print hzurl, num
                if not postdata:
                    selecotr.to_mq(hzurl, None,method)
                    print hzurl
    except getopt.GetoptError:
        print traceback.format_exc()
        sys.exit()

if __name__ == '__main__':
    op = sys.argv[1]
    if op == 'rule':
        rule_sheduler()
    else:
        file_sheduler()
